package com.otaku.controller;

import org.apache.log4j.Logger;

import com.otaku.controller.pool.ControllerPool;
import com.otaku.controller.pool.MyCrawlerController;
import com.otaku.crawler.MTimeCrawler;
import com.otaku.crawler.NASaveMoneyCrawler;
import com.otaku.crawler.USAShopCNCrawler;
import com.otaku.crawler.WAHTCrawler;
import com.otaku.crawler.util.EmailHelper;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

public class Controller {
	private static final Logger log = Logger.getRootLogger();
	
	public static void main(String[] args){
		log.warn("begin controller");
		try {
//			String[] NASaveSeeds = new String[]{"http://www.dealmoon.com/cn/"};
//			String[] WAHTSeeds = new String[]{"http://www.woaihaitao.com/"};
//			
//			addController(5, NASaveSeeds, NASaveMoneyCrawler.class,"D://crawler info//NASave");
//			addController(5, WAHTSeeds, WAHTCrawler.class,"D://crawler info//WAHT");
			
			String[] MTimeSeeds = new String[]{"http://theater.mtime.com"};
			addController(1, MTimeSeeds, MTimeCrawler.class, "D://crawler info//MTime");
			
			ControllerPool.run();
		} catch (Exception e) {
			EmailHelper.sendMsg("controller caught exception", e);
			e.printStackTrace();
		}

	}
	
	private static <T extends WebCrawler> void addController(int threadNum,String[] seeds,
			Class<T> crawlerClass,String storageFolder) throws Exception{
		CrawlConfig config = new CrawlConfig();
		config.setCrawlStorageFolder(storageFolder);
		config.setPolitenessDelay(8000);

		/*
		 * Instantiate the controller for this crawl.
		 */
		PageFetcher pageFetcher = new PageFetcher(config);
		RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
		RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
		CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
		
		/*
		 * For each crawl, you need to add some seed urls. These are the first
		 * URLs that are fetched and then the crawler starts following links
		 * which are found in these pages
		 */
		for(String s : seeds){
			controller.addSeed(s);
		}
		
		ControllerPool.addController(new MyCrawlerController(controller, crawlerClass, threadNum));
	}
}
